package com.spark.statics

import org.apache.spark.mllib.linalg
import org.apache.spark.mllib.linalg.{Matrix, Vectors}
import org.apache.spark.mllib.stat.{MultivariateStatisticalSummary, Statistics}
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by TRS on 2017/6/27.
  */
object statics {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("statics").setMaster("local")
    val sc = new SparkContext(conf)

    val observations = sc.parallelize(
      Seq(
        Vectors.dense(1.0, 10.0),
        Vectors.dense(2.0, 20.0),
        Vectors.dense(3.0, 30.0)
      )
    )

    // 计算概要统计.
    val summary: MultivariateStatisticalSummary = Statistics.colStats(observations)
    println(summary.mean) //  每一列的均值
    println(summary.variance) // 所有向量的方差
    println(summary.numNonzeros) // 每列中的非零数



    val seriesX: RDD[Double] = sc.parallelize(Array(1, 2, 3, 3, 5))  // a series
    // 必须具有相同数量的分区和基数作为一个series
    val seriesY: RDD[Double] = sc.parallelize(Array(11, 22, 33, 33, 555))

    // 计算 Pearson's 相关性.输入"spearman" 调用 Spearman's 的计算方法.
    // 默认使用 Pearson's 方法.
    val correlation: Double = Statistics.corr(seriesX, seriesY, "pearson")
    println(s"Correlation is: $correlation")

    val data: RDD[linalg.Vector] = sc.parallelize(
      Seq(
        Vectors.dense(1.0, 10.0, 100.0),
        Vectors.dense(2.0, 20.0, 200.0),
        Vectors.dense(5.0, 33.0, 366.0))
    )  // 注意每个向量为行而不是列
    // 计算 Pearson's 相关性的矩阵.输入"spearman" 调用 Spearman's 的计算方法.
    // 默认使用 Pearson's 方法.
    val correlMatrix: Matrix = Statistics.corr(data, "pearson")
    println(correlMatrix.toString)
  }

}
